package com.offcn.spark.p3

import org.apache.spark.{SparkConf, SparkContext}

/**
 * @Auther: BigData-LGW
 * @ClassName: FoldBy
 * @Date: 2020/12/7 19:55
 * @功能描述: $FunctionDescription
 * @Version:1.0
 */
object FoldBy {
    def main(args: Array[String]): Unit = {
        val conf = new SparkConf()
            .setMaster("local[*]")
            .setAppName("FoldBy")
        val sc = new SparkContext(conf)
        foldBy(sc)
        sc.stop()
    }
    def foldBy(sc: SparkContext) = {
        case class Student(id:Int,name:String,province:String)
        val stuRDD = sc.parallelize(List(
            Student(1, "白普州", "安徽"),
            Student(2, "伍齐城", "山东"),
            Student(3, "曹佳", "甘肃"),
            Student(4, "姚远", "甘肃"),
            Student(5, "匿名大哥", "黑吉辽"),
            Student(10086, "欧阳龙生", "黑吉辽")
        ),2).mapPartitionsWithIndex((index,partition) => {
            val list = partition.toList
            println(s"-->stuRDD的分区编号为<${index}>中的数据为：${list.mkString("[", ", ", "]")}")
            list.toIterator
        })
        val ret = stuRDD.map(stu => (stu.province,1)).foldByKey(0)((v1,v2) => v1 + v2)
        ret.foreach{
            case (province,count) => {
                println(s"province: ${province}, count: ${count}")
            }
        }
    }

}
